Credit Card Fraud Detection

• Packages

• Data

  • Data overview

• Exploratory Data Analysis

  • Fraud and Valid Transactions in data
  • Varible Summary
  • Histograms of all variables
  • Scatter Plots
  • Correlation Matrix
  • Monthly charges,total charges and tenure in customer attrition
  • Variable Summary
  • Correlation Matrix

• Imbalanced Data

  • Model Building

    • Logistic Regression (Baseline model)
    • KNN Classifier
    • Random forest classifier
    • Gaussian Naive Bayes
    • XGBoost Classifier

  • Model Evaluation

    • model performance metrics
    • Compare model metrics
    • Confusion matrices for models
    • ROC - Curves for models
    • Precision recall curves

• Unsupervised Outlier Detection

  • Local Outlier Factor (LOF)
  • Isolation Forest Algorithm
    • Model Building and Performance
    • Confusion Matrix
    • Compare Models Performance

Packages

import sys
import numpy
import matplotlib
import pandas
import plotly
import seaborn
import sklearn
import scipy
 
print(f"""
Python - {sys.version}
Numpy - {numpy.__version__}
Matplotlib - {matplotlib.__version__}
Pandas - {pandas.__version__}
Plotly - {plotly.__version__}
Seaborn - {seaborn.__version__}
Sklearn - {sklearn.__version__}
Scipy - {scipy.__version__}
""")

Python - 3.8.5 (default, Jul 28 2020, 12:59:40) 
[GCC 9.3.0]
Numpy - 1.19.2
Matplotlib - 3.3.2
Pandas - 1.1.3
Plotly - 4.11.0
Seaborn - 0.11.0
Sklearn - 0.23.2
Scipy - 1.5.3

# Importing the neccessary libraries
from collections import Counter 
import numpy as np
import matplotlib.pyplot as plt 
import pandas as pd
import random
 
import plotly
import plotly.offline as py
import plotly.graph_objs as go
import plotly.figure_factory as ff
import plotly.express as px
import plotly.tools as tls
 
import seaborn as sns
import sklearn
import scipy
import itertools 

 
import warnings
warnings.filterwarnings("ignore")

Data

#import the Dataset
df = pd.read_csv('/home/abhishek/Documents/Jupyter Notebooks/creditcardML-master/creditcard.csv')
df.head()

	Time	V1	V2	V3	V4	V5	V6	V7	V8	V9	...	V21	V22	V23	V24	V25	V26	V27	V28	Amount	Class
0	0.0	-1.359807	-0.072781	2.536347	1.378155	-0.338321	0.462388	0.239599	0.098698	0.363787	...	-0.018307	0.277838	-0.110474	0.066928	0.128539	-0.189115	0.133558	-0.021053	149.62	0
1	0.0	1.191857	0.266151	0.166480	0.448154	0.060018	-0.082361	-0.078803	0.085102	-0.255425	...	-0.225775	-0.638672	0.101288	-0.339846	0.167170	0.125895	-0.008983	0.014724	2.69	0
2	1.0	-1.358354	-1.340163	1.773209	0.379780	-0.503198	1.800499	0.791461	0.247676	-1.514654	...	0.247998	0.771679	0.909412	-0.689281	-0.327642	-0.139097	-0.055353	-0.059752	378.66	0
3	1.0	-0.966272	-0.185226	1.792993	-0.863291	-0.010309	1.247203	0.237609	0.377436	-1.387024	...	-0.108300	0.005274	-0.190321	-1.175575	0.647376	-0.221929	0.062723	0.061458	123.50	0
4	2.0	-1.158233	0.877737	1.548718	0.403034	-0.407193	0.095921	0.592941	-0.270533	0.817739	...	-0.009431	0.798278	-0.137458	0.141267	-0.206010	0.502292	0.219422	0.215153	69.99	0

5 rows × 31 columns

• Due to confidentiality issue, original features V1, V2, V3, ... V28 have been transformed with PCA.
• The only features which have not been transformed with PCA are 'Time' and 'Amount'.
• Feature Time contains the seconds elapsed between each transaction and the first transaction in the dataset. The feature Amount is the transaction Amount.
• Feature Class is response variable and 1 and 0 indicates Fraud and Valid Transactions respectively.

Data Overview

print ("Rows     : " ,df.shape[0])
print ("Columns  : " ,df.shape[1])
print ("\nFeatures : \n" ,df.columns.tolist())
print ("\nMissing values :  ", df.isnull().sum().values.sum())
print ("\nUnique values :  \n",df.nunique())

Rows     :  284807
Columns  :  31

Features : 
 ['Time', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28', 'Amount', 'Class']

Missing values :   0

Unique values :  
 Time      124592
V1        275663
V2        275663
V3        275663
V4        275663
V5        275663
V6        275663
V7        275663
V8        275663
V9        275663
V10       275663
V11       275663
V12       275663
V13       275663
V14       275663
V15       275663
V16       275663
V17       275663
V18       275663
V19       275663
V20       275663
V21       275663
V22       275663
V23       275663
V24       275663
V25       275663
V26       275663
V27       275663
V28       275663
Amount     32767
Class          2
dtype: int64

• Dataset contains 284807 rows (transactions) with 31 columns (features).
• No missing data in our dataset.
• All data values are float64, except from column Class.

Exploratory Data *Analysis*

Fraud and Valid Transactions in Data

#labels
lab = df["Class"].value_counts().keys().tolist()
#values
val = df["Class"].value_counts().values.tolist()
 
trace = go.Pie(labels = ['Valid Transactions','Fraud Transactions'] ,
               values = val ,
               marker = dict(colors =  ["lime", "orangered"],
                             line = dict(color = "white",
                                         width =  1.3)
                            ),
               rotation = 90,
               hoverinfo = "label+value+text",
               hole = .5
              )
layout = go.Layout(dict(title = "<b>Fraud and Valid Transactions<b>",
                        height=400,
                        width=700,
                        plot_bgcolor  = "rgb(243,243,243)",
                        paper_bgcolor = "rgb(243,243,243)",
                       )
                  )
 
data = [trace]
fig = go.Figure(data = data,layout = layout)
py.iplot(fig)

The Dataset contains 492 frauds out of 284,315 transactions. The dataset is highly unbalanced, the positive class (frauds) account for 0.173% of all transactions.

We can see most of the transactions are valid transactions.It shows that it's real world dataset. If we use this dataframe as the base for our predictive models and analysis, our algorithms will probably overfit since it will "assume" that most transactions are not a fraud.

Variable Summary

summary = df.describe().T.reset_index()
summary = summary.rename(columns = {"index" : "feature"})
summary = np.around(summary,3)

val_lst = [summary['feature'], summary['count'],
           summary['mean'],summary['std'],
           summary['min'], summary['25%'],
           summary['50%'], summary['75%'], summary['max']]

trace  = go.Table(header = dict(values = summary.columns.tolist(),
                                line = dict(color = ['#506784']),
                                fill = dict(color = ['#119DFF']),
                               ),
                  cells  = dict(values = val_lst,
                                line = dict(color = ['#506784']),
                                fill = dict(color = ["lightgrey",'#F5F8FF'])
                               ),
                  columnwidth = [200,60,100,100,60,60,80,80,80])
layout = go.Layout(dict(title = "<b>Variable Summary<b>",height=850,width=1000))
figure = go.Figure(data=[trace],layout=layout)
py.iplot(figure)

In all the features only Time, Amount, and Class (fraud or not fraud) are made the most sense.The other 28 columns were transformed using what seems to be a PCA (Dimensionality Reduction technique) in order to protect user identities.

fraud = df[df['Class']==1]
valid = df[df['Class']==0]
fraud.Amount.describe()

count     492.000000
mean      122.211321
std       256.683288
min         0.000000
25%         1.000000
50%         9.250000
75%       105.890000
max      2125.870000
Name: Amount, dtype: float64

Histograms of all Variables

df.hist(figsize=(30,30),color='#4FC3F7')
plt.show()

By seeing the 1st graph which is of variable Time , we can see there are two peaks in the graph.

These as the time of the day like the peak is the day time when most people do the transactions and the depth is the night time when most people just sleeps. Because the data contains a credit card transaction for only two days, so there are two peaks for day time and one depth for one night time.

Scatter Plots

def scatter_matrix(df)  :
    
    df  = df.sort_values(by = "Class" ,ascending = True)
    classes = df["Class"].unique().tolist()
 
    class_code  = {classes[k] : k for k in range(2)}
 
    color_vals = [class_code[cl] for cl in df["Class"]]
 
    pl_colorscale = np.random.choice(['Bluered','Earth','Electric','Hot','Jet','Picnic','Portland','Rainbow','RdBu',
                    'Viridis','YlGnBu','YlOrRd','magma','inferno','Plasma','Blackbody'])
 
    text = [df.loc[k,"Class"] for k in range(len(df))]
 
    trace = go.Splom(dimensions = [ dict(label = f"{i}", values = df[f"{i}"]) for i in ["Time","Amount","V1","V2","V3"]],
                     text = text,
                     showupperhalf=False, # remove plots on diagonal
                     marker = dict(color = color_vals,
                                   colorscale = ["#FFC107 ","#FF5722"],
                                   size = 3,
                                   showscale = False,
                                   line = dict(width = .1,
                                               color='rgb(230,230,230)'
                                              )
                                  )
                    )
    axis = dict(showline  = True,
                zeroline  = False,
                gridcolor = "#fff",
                ticklen   = 4
               )
    
    layout = go.Layout(dict(title  = 
                            "Scatter plot",
                            autosize = False,
                            height = 1080,
                            width  = 1080,
                            dragmode = "select",
                            hovermode = "closest",
                            plot_bgcolor  = 'rgba(240,240,240, 0.95)',
                            xaxis1 = dict(axis),
                            yaxis1 = dict(axis),
                            xaxis2 = dict(axis),
                            yaxis2 = dict(axis),
                            xaxis3 = dict(axis),
                            yaxis3 = dict(axis),
                           )
                      )
    data   = [trace]
    fig = go.Figure(data = data,layout = layout )
    py.iplot(fig)
 
scatter_matrix(df)

Correlation Matrix

Find out top features which are highly correlated.

fig = ff.create_annotated_heatmap(round(df.corr(),1).values.tolist(),
                                  
                                  y=df.columns.tolist(),
                                  x=df.columns.tolist(), 
                                  colorscale="Viridis",
                                  showscale=True
                                 )
for i in range(len(fig.layout.annotations)):
    fig.layout.annotations[i].font.size = 7

fig.show()

We can see there are lots of features in this model but having lots of feature doesn't always helps to the model, Instead of we might get in overfitting issues and get bad results.

**Imbalanced Data**

# Seperating Independant and Dependent Variables
# Get all the columns from the dataFrame
columns_ = df.columns.tolist()

# Filter the columns to remove data we do not want i.e target variable "Class"
features = [c for c in columns_ if c not in ["Class"]]

# Store the variable we'll be predicting on
target = "Class"

X = df[features];print("Features (X): ",X.shape)
Y = df[target];print("Target (Y): ",Y.shape)

Features (X):  (284807, 30)
Target (Y):  (284807,)

Model Building

from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix,accuracy_score,classification_report
from sklearn.metrics import roc_auc_score,roc_curve,scorer
from sklearn.metrics import f1_score
import statsmodels.api as sm
from sklearn.metrics import precision_score,recall_score
from yellowbrick.classifier import DiscriminationThreshold

# Splitting the data
x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.25, shuffle=True, random_state=0)

print(f"""
X_train: {x_train.shape}
y_train: {y_train.shape}
X_test: {x_test.shape}
y_test: {y_test.shape}""")

def Class_Transactions(alg_nam,algorithm,training_x,testing_x,
                             training_y,testing_y,threshold_plot,):
    """
    Attributes:
    alg_nam          - Algorithm Name (str)
    algorithm        - Algorithm used 
    training_x       - predictor variables dataframe(training)
    testing_x        - predictor variables dataframe(testing)
    training_y       - target variable(training)
    training_y       - target variable(testing)
    threshold_plot   - if True returns threshold plot for model
    """
    #model
    algorithm.fit(training_x,training_y)
    predictions   = algorithm.predict(testing_x)
    probabilities = algorithm.predict_proba(testing_x)

    print (algorithm)
    print(f"\nActual testing transactions :  {Counter(testing_y)}\nPredicted testing transactions :  {Counter(predictions)}")
    print("\nClassification report : \n\n",classification_report(testing_y,predictions))
    print("Accuracy   Score : ",accuracy_score(testing_y,predictions))

    #confusion matrix
    conf_matrix = confusion_matrix(testing_y,predictions)

    #roc_auc_score
    model_roc_auc = roc_auc_score(testing_y,predictions) 
    print ("Area under curve : ",model_roc_auc,"\n")
    fpr,tpr,thresholds = roc_curve(testing_y,probabilities[:,1])
    
    #plot confusion matrix
    trace1 = go.Heatmap(z = conf_matrix ,
                        x = ["Valid","Fraud"],
                        y = ["Valid","Fraud"],
                        showscale  = False,
                        colorscale = np.random.choice(['Bluered','Earth','Electric','Hot','Jet','Picnic','Portland','Rainbow','RdBu',
                    'Viridis','YlGnBu','YlOrRd','magma','inferno','Plasma','Blackbody']),
                        name="Transactions")
    
    #plot roc curve
    trace2 = go.Scatter(x = fpr,y = tpr,
                        name = "Roc : " + str(model_roc_auc),
                        line = dict(color = ('rgb(22, 96, 167)'),width = 2))
    trace3 = go.Scatter(x = [0,1],y=[0,1],
                        line = dict(color = ('rgb(205, 12, 24)'),width = 2,
                        dash = 'dot'))

    #subplots
    fig = plotly.subplots.make_subplots(rows=1, cols=2,
                            subplot_titles=('Confusion Matrix',
                                            'Receiver operating characteristic (ROC)'))
    fig.append_trace(trace1,1,1)
    fig.append_trace(trace2,1,2)
    fig.append_trace(trace3,1,2)


    fig["layout"]["yaxis1"].update(dict(title = "Predicted lables"))
    fig["layout"]["xaxis1"].update(dict(title = "Actual lables"))

    fig['layout'].update(showlegend=False, title=f"<b>Model Performance for {alg_nam}<b>" ,
                         autosize = False,
                         height=500,
                         width=900,
                         plot_bgcolor = 'rgba(240,240,240, 0.95)',
                         paper_bgcolor = 'rgba(240,240,240, 0.95)',
                         )
    
    fig["layout"]["xaxis2"].update(dict(title = "False Positive Rate"))
    fig["layout"]["yaxis2"].update(dict(title = "True Positive Rate"))

    py.iplot(fig)
    
    if threshold_plot == True: 
        visualizer = DiscriminationThreshold(algorithm)
        visualizer.fit(training_x,training_y)
        visualizer.poof()

    ##### End of the Function #####
    

X_train: (213605, 30)
y_train: (213605,)
X_test: (71202, 30)
y_test: (71202,)

Logistic Regression

from sklearn.linear_model import LogisticRegression

logit  = LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,
          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,
          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,
          verbose=0, warm_start=False)

Class_Transactions("Logistic Regression",logit,x_train,x_test,y_train,y_test,threshold_plot = True)

LogisticRegression(multi_class='ovr', n_jobs=1, solver='liblinear')

Actual testing transactions :  Counter({0: 71082, 1: 120})
Predicted testing transactions :  Counter({0: 71113, 1: 89})

Classification report : 

               precision    recall  f1-score   support

           0       1.00      1.00      1.00     71082
           1       0.81      0.60      0.69       120

    accuracy                           1.00     71202
   macro avg       0.90      0.80      0.84     71202
weighted avg       1.00      1.00      1.00     71202

Accuracy   Score :  0.9990871042948232
Area under curve :  0.7998804197968543 

KNN Classifier

from sklearn.neighbors import KNeighborsClassifier
knn = KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',
           metric_params=None, n_jobs=1, n_neighbors=5, p=2,
           weights='uniform')
Class_Transactions("KNN Classifier",knn,x_train,x_test,y_train,y_test,threshold_plot = True)

KNeighborsClassifier(n_jobs=1)

Actual testing transactions :  Counter({0: 71082, 1: 120})
Predicted testing transactions :  Counter({0: 71196, 1: 6})

Classification report : 

               precision    recall  f1-score   support

           0       1.00      1.00      1.00     71082
           1       0.83      0.04      0.08       120

    accuracy                           1.00     71202
   macro avg       0.92      0.52      0.54     71202
weighted avg       1.00      1.00      1.00     71202

Accuracy   Score :  0.9983708322799921
Area under curve :  0.5208262992037366 

Random Forest Classifier

from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import export_graphviz
from sklearn import tree
from graphviz import Source
from IPython.display import SVG,display




def plot_tree_randomforest(columns,nf_estimators,
                           estimated_tree,maximum_depth,
                           criterion_type):
    """
    Attributes
    columns  : column used
    nf_estimators   : The number of trees in the forest.
    estimated_tree  : tree number to be displayed
    maximum_depth   : depth of the tree
    criterion_type  : split criterion type ["gini" or "entropy"]
    """

    
    #train and test datasets
    rf_x     = x_train
    rf_y     = y_train
    
    #random forest classifier
    rfc   = RandomForestClassifier(n_estimators = nf_estimators,
                                   max_depth = maximum_depth,
                                   criterion = criterion_type,
                                  )
    rfc.fit(rf_x,rf_y)
    
    estimated_tree = rfc.estimators_[estimated_tree]
    
    graph = Source(tree.export_graphviz(estimated_tree,out_file=None,
                                        rounded=True,proportion = False,
                            feature_names = columns, 
                            precision  = 2,
                            class_names=["Valid","Fraud"],
                            filled = True))
    display(graph)
    
    ### End ###
 
plot_tree_randomforest(X.columns,100,99,3,"entropy")

rfc  = RandomForestClassifier(n_estimators = 100,
                               max_depth = 99,
                               criterion = "entropy")

Class_Transactions("Random Forest",rfc,x_train,x_test,y_train,y_test,threshold_plot = False)

RandomForestClassifier(criterion='entropy', max_depth=99)

Actual testing transactions :  Counter({0: 71082, 1: 120})
Predicted testing transactions :  Counter({0: 71103, 1: 99})

Classification report : 

               precision    recall  f1-score   support

           0       1.00      1.00      1.00     71082
           1       0.94      0.78      0.85       120

    accuracy                           1.00     71202
   macro avg       0.97      0.89      0.92     71202
weighted avg       1.00      1.00      1.00     71202

Accuracy   Score :  0.9995365298727564
Area under curve :  0.8874577952224192 

• A random forest is a meta estimator that fits a number of decision tree classifiers on various sub-samples of the dataset and use averaging to improve the predictive accuracy and control over-fitting. The sub-sample size is always the same as the original input sample size but the samples are drawn with replacement .
• Below are the trees produced by random forest model with 10 estimated trees with maximum depth of three for each tree. Each tree produced is slightly different from other.

#making 10 trees with random forest.
n = np.arange(0,10).tolist()
for i in n :
    print(f"\n                              #### [{i+1}] #### \n")
    plot_tree_randomforest(x_train.columns,10,i,3,"entropy")

                              #### [1] #### 

                              #### [2] #### 

                              #### [3] #### 

                              #### [4] #### 

                              #### [5] #### 

                              #### [6] #### 

                              #### [7] #### 

                              #### [8] #### 

                              #### [9] #### 

                              #### [10] #### 

Gaussian Naive Bayes

from sklearn.naive_bayes import GaussianNB
gnb = GaussianNB(priors=None)

Class_Transactions("Gaussian Naive Bayes",gnb,x_train,x_test,y_train,y_test,threshold_plot=True)

GaussianNB()

Actual testing transactions :  Counter({0: 71082, 1: 120})
Predicted testing transactions :  Counter({0: 70677, 1: 525})

Classification report : 

               precision    recall  f1-score   support

           0       1.00      0.99      1.00     71082
           1       0.14      0.63      0.24       120

    accuracy                           0.99     71202
   macro avg       0.57      0.81      0.62     71202
weighted avg       1.00      0.99      1.00     71202

Accuracy   Score :  0.9930760371899666
Area under curve :  0.8135083424777019 

XGBoost Classifier

from xgboost import XGBClassifier

xgc = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
                    colsample_bytree=1, gamma=0, learning_rate=0.9, max_delta_step=0,
                    max_depth = 7, min_child_weight=1, missing=None, n_estimators=100,
                    n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,
                    reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
                    silent=True, subsample=1)


Class_Transactions("XGBoost Classifier",xgc,x_train,x_test,y_train,y_test,threshold_plot = False)

[18:06:21] WARNING: ../src/learner.cc:516: 
Parameters: { silent } might not be used.

  This may not be accurate due to some parameters are only used in language bindings but
  passed down to XGBoost core.  Or some parameters are not used but slip through this
  verification. Please open an issue if you find above cases.


XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
              colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,
              importance_type='gain', interaction_constraints='',
              learning_rate=0.9, max_delta_step=0, max_depth=7,
              min_child_weight=1, missing=None, monotone_constraints='()',
              n_estimators=100, n_jobs=1, nthread=1, num_parallel_tree=1,
              random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,
              seed=0, silent=True, subsample=1, tree_method='exact',
              validate_parameters=1, verbosity=None)

Actual testing transactions :  Counter({0: 71082, 1: 120})
Predicted testing transactions :  Counter({0: 71099, 1: 103})

Classification report : 

               precision    recall  f1-score   support

           0       1.00      1.00      1.00     71082
           1       0.94      0.81      0.87       120

    accuracy                           1.00     71202
   macro avg       0.97      0.90      0.93     71202
weighted avg       1.00      1.00      1.00     71202

Accuracy   Score :  0.999592708069998
Area under curve :  0.9041244618890858 

Model Evaluation

Model performance metrics

from sklearn.metrics import f1_score
from sklearn.metrics import cohen_kappa_score

#Gives model report in dataframe
def model_report(model,training_x,testing_x,training_y,testing_y,name) :
    model.fit(training_x,training_y)
    predictions  = model.predict(testing_x)
    
    accuracy     = accuracy_score(testing_y,predictions)
    recallscore  = recall_score(testing_y,predictions)
    precision    = precision_score(testing_y,predictions)
    roc_auc      = roc_auc_score(testing_y,predictions)
    f1score      = f1_score(testing_y,predictions) 
    kappa_metric = cohen_kappa_score(testing_y,predictions)
    
    df = pd.DataFrame({"Model"           : [name],
                       "Accuracy_score"  : [accuracy],
                       "Recall_score"    : [recallscore],
                       "Precision"       : [precision],
                       "f1_score"        : [f1score],
                       "Area_under_curve": [roc_auc],
                       "Kappa_metric"    : [kappa_metric],
                      })
    return df

#outputs for every model
model1 = model_report(logit,x_train, x_test, y_train, y_test,
                      "Logistic Regression(Baseline_model)")
model2 = model_report(knn,x_train, x_test, y_train, y_test,
                      "KNN Classifier")
rfc  = RandomForestClassifier(n_estimators = 100,
                               max_depth = 99,
                               criterion = "entropy")
model3 = model_report(rfc,x_train, x_test, y_train, y_test,
                      "Random Forest Classifier")
model4 = model_report(gnb,x_train, x_test, y_train, y_test,
                      "Gaussian Naive Bayes")
model5 = model_report(xgc,x_train, x_test, y_train, y_test,
                      "XGBoost Classifier")

#concat all models
model_performances = pd.concat([model1,model2,model3,
                                model4,model5],axis = 0).reset_index()

model_performances = model_performances.drop(columns = "index",axis =1)

table  = ff.create_table(np.round(model_performances,4))

py.iplot(table)

[18:12:23] WARNING: ../src/learner.cc:516: 
Parameters: { silent } might not be used.

  This may not be accurate due to some parameters are only used in language bindings but
  passed down to XGBoost core.  Or some parameters are not used but slip through this
  verification. Please open an issue if you find above cases.

Compare Model Metrics

model_performances
def output_tracer(metric,color) :
    tracer = go.Bar(y = model_performances["Model"] ,
                    x = model_performances[metric],
                    orientation = "h",name = metric ,
                    marker = dict(line = dict(width =.7),
                                  color = color)
                   )
    return tracer

layout = go.Layout(dict(title = "Model Performances",
                        plot_bgcolor  = "rgb(243,243,243)",
                        paper_bgcolor = "rgb(243,243,243)",
                        xaxis = dict(gridcolor = 'rgb(255, 255, 255)',
                                     title = "metric",
                                     zerolinewidth=1,
                                     ticklen=5,gridwidth=2),
                        yaxis = dict(gridcolor = 'rgb(255, 255, 255)',
                                     zerolinewidth=1,ticklen=5,gridwidth=2),
                        margin = dict(l = 250),
                        height = 780
                       )
                  )

trace1  = output_tracer("Accuracy_score","#6699FF")
trace2  = output_tracer('Recall_score',"red")
trace3  = output_tracer('Precision',"#33CC99")
trace4  = output_tracer('f1_score',"lightgrey")
trace5  = output_tracer('Kappa_metric',"#FFCC99")

data = [trace1,trace2,trace3,trace4,trace5]
fig = go.Figure(data=data,layout=layout)
py.iplot(fig)

Receiver Operating Characteristic (ROC) - Curves

lst    = [logit, knn, rfc, gnb, xgc]

length = len(lst)

mods   = ['Logistic Regression(Baseline_model)','KNN Classifier','Random Forest Classifier',"Gaussian Naive Bayes",'XGBoost Classifier']

plt.style.use("dark_background")
fig = plt.figure(figsize=(12,16))
fig.set_facecolor("#F3F3F3")
for i,j,k in itertools.zip_longest(lst,range(length),mods) :
    qx = plt.subplot(4,3,j+1)
    probabilities = i.predict_proba(x_test)
    predictions   = i.predict(x_test)
    fpr,tpr,thresholds = roc_curve(y_test,probabilities[:,1])
    plt.plot(fpr,tpr,linestyle = "dotted",
             color = "royalblue",linewidth = 2,
             label = "AUC = " + str(np.around(roc_auc_score(y_test,predictions),3)))
    plt.plot([0,1],[0,1],linestyle = "dashed",
             color = "orangered",linewidth = 1.5)
    plt.fill_between(fpr,tpr,alpha = .4)
    plt.fill_between([0,1],[0,1],color = "k")
    plt.legend(loc = "lower right",
               prop = {"size" : 12})
    qx.set_facecolor("k")
    plt.grid(True,alpha = .15)
    plt.title(k,color = "b")
    plt.xticks(np.arange(0,1,.3))
    plt.yticks(np.arange(0,1,.3))

Precision Recall Curves

from sklearn.metrics import precision_recall_curve
from sklearn.metrics import average_precision_score

lst    = [logit, knn, rfc, gnb, xgc]

length = len(lst)

mods   = ['Logistic Regression(Baseline_model)','KNN Classifier','Random Forest Classifier',"Gaussian Naive Bayes",'XGBoost Classifier']

fig = plt.figure(figsize=(13,17))
fig.set_facecolor("#F3F3F3")
for i,j,k in itertools.zip_longest(lst,range(length),mods) :
    
    qx = plt.subplot(4,3,j+1)
    probabilities = i.predict_proba(x_test)
    predictions   = i.predict(x_test)
    recall,precision,thresholds = precision_recall_curve(y_test,probabilities[:,1])
    plt.plot(recall,precision,linewidth = 1.5,
             label = ("avg_pcn : " + 
                      str(np.around(average_precision_score(y_test,predictions),3))))
    plt.plot([0,1],[0,0],linestyle = "dashed")
    plt.fill_between(recall,precision,alpha = .2)
    plt.legend(loc = "lower left",
               prop = {"size" : 10})
    qx.set_facecolor("k")
    plt.grid(True,alpha = .15)
    plt.title(k,color = "b")
    plt.xlabel("recall",fontsize =7)
    plt.ylabel("precision",fontsize =7)
    plt.xlim([0.25,1])
    plt.yticks(np.arange(0,1,.3))
    

**Unsupervised Outlier Detection**

Now that we have processed our data, we can begin deploying our machine learning algorithms. We will use the following techniques:

Local Outlier Factor (LOF)

The anomaly score of each sample is called Local Outlier Factor. It measures the local deviation of density of a given sample with respect to its neighbors. It is local in that the anomaly score depends on how isolated the object is with respect to the surrounding neighborhood.

Isolation Forest Algorithm

The IsolationForest ‘isolates’ observations by randomly selecting a feature and then randomly selecting a split value between the maximum and minimum values of the selected feature.

Since recursive partitioning can be represented by a tree structure, the number of splittings required to isolate a sample is equivalent to the path length from the root node to the terminating node.

This path length, averaged over a forest of such random trees, is a measure of normality and our decision function.

Random partitioning produces noticeably shorter paths for anomalies. Hence, when a forest of random trees collectively produce shorter path lengths for particular samples, they are highly likely to be anomalies.

Model Building and Performance

from sklearn.ensemble import IsolationForest
from sklearn.neighbors import LocalOutlierFactor
from sklearn.svm import OneClassSVM

# Define random states
state = 1

fraud = y_train[y_train.values == 1]
valid = y_train[y_train.values == 0]
print(f"Fraud : {fraud.shape[0]}\nValid : {valid.shape[0]}")

outlier_fraction = fraud.count()/valid.count()

# Define outlier detection tools to be compared
classifiers = {
    "Isolation Forest": IsolationForest(max_samples=len(X),
                                        contamination=outlier_fraction,
                                        random_state=state),

    "Local Outlier Factor": LocalOutlierFactor(
        n_neighbors=20,
        contamination=outlier_fraction),
        

    "One Class SVM":OneClassSVM(
        kernel='rbf', 
        degree=3, 
        gamma=0.1,
        nu=0.05,
        max_iter=-1)

    }

Fraud : 372
Valid : 213233

from sklearn.metrics import f1_score, confusion_matrix, cohen_kappa_score

#Gives model report in dataframe
def model_report(model,training,testing,model_name):
    global conf_matrix1,conf_matrix2
    n_outliers = len(fraud)

    if model_name == "Local Outlier Factor":
        predictions = model.fit_predict(training)
        # scores_pred = model.negative_outlier_factor_
        # print(scores_pred)

    # if model_name == "One Class SVM":
    #     predictions = model.fit(training)
    #     predictions = model.predict(training)
    
    if model_name == "Isolation Forest":
        predictions = model.fit_predict(training)
        # scores_pred = model.decision_function(training)
        
    #Reshape the prediction values to 0 for valid, 1 for fraud. 
    predictions[predictions == 1] = 0
    predictions[predictions == -1] = 1
    
    conf_mat = confusion_matrix(testing, predictions)

    #Error - Total Number of all wrong predictions by a model
    n_errors = (predictions != testing).sum()

    accuracy     = accuracy_score(testing,predictions)
    recallscore  = recall_score(testing,predictions)
    precision    = precision_score(testing,predictions)
    roc_auc      = roc_auc_score(testing,predictions)
    f1score      = f1_score(testing,predictions) 
    kappa_metric = cohen_kappa_score(testing,predictions)


    df = pd.DataFrame({"Model"           : [model_name],
                       "Errors"          : [n_errors],
                       "Accuracy_score"  : [accuracy],
                       "Recall_score"    : [recallscore],
                       "Precision"       : [precision],
                       "f1_score"        : [f1score],
                       "Area_under_curve": [roc_auc],
                       "Kappa_metric"    : [kappa_metric],
                      })
    return df,conf_mat


#outputs for every model
model1, conf_matrix1 = model_report(classifiers["Local Outlier Factor"], x_train, y_train, "Local Outlier Factor")
model2,  conf_matrix2 = model_report(classifiers["Isolation Forest"], x_train, y_train, "Isolation Forest")
# model3, conf_matrix3 = model_report(classifiers["One Class SVM"], x_train, y_train, "One Class SVM")

#concat all models
model_performances = pd.concat([model1,model2], axis = 0).reset_index()

model_performances = model_performances.drop(columns = "index",axis =1)

table  = ff.create_table(np.round(model_performances,4))

py.iplot(table)

Confuion Matrix

#PLOT CONFUSION MATRIX

clr_scale_list=['Bluered','Earth','Electric','Hot','Jet','Picnic','Portland','Rainbow','RdBu',
                    'Viridis','YlGnBu','YlOrRd','magma','inferno','Plasma','Blackbody']
trace1 = go.Heatmap(z = conf_matrix1 ,
                    x = ["Valid","Fraud"],
                    y = ["Valid","Fraud"],
                    showscale  = False,
                    colorscale = np.random.choice(clr_scale_list),
                    name="Transactions")
trace2 = go.Heatmap(z = conf_matrix2 ,
                x = ["Valid","Fraud"],
                y = ["Valid","Fraud"],
                showscale  = False,
                colorscale = np.random.choice(clr_scale_list),
                name="Transactions")

#subplots for both models
fig = plotly.subplots.make_subplots(rows=1, cols=2,
                        subplot_titles=('Local Outlier Factor',
                                        'Isolation Forest'))
fig.append_trace(trace1,1,1)
fig.append_trace(trace2,1,2)

fig["layout"]["yaxis1"].update(dict(title = "Predicted lables"))
fig["layout"]["yaxis2"].update(dict(title = "Predicted lables"))
fig["layout"]["xaxis1"].update(dict(title = "Actual lables"))
fig["layout"]["xaxis2"].update(dict(title = "Actual lables"))

fig['layout'].update(showlegend=False,
                        title=f"<b>Confusion Matrices<b>" ,
                        autosize = False,
                        height=500,
                        width=900,
                        plot_bgcolor = 'rgba(240,240,240, 0.95)',
                        paper_bgcolor = 'rgba(240,240,240, 0.95)',
                        )
py.iplot(fig)

Compare Models

model_performances
def output_tracer(metric,color) :
    tracer = go.Bar(y = model_performances["Model"] ,
                    x = model_performances[metric],
                    orientation = "h",name = metric ,
                    marker = dict(line = dict(width =.7),
                                  color = color)
                   )
    return tracer

layout = go.Layout(dict(title = "Compare Model performances",
                        plot_bgcolor  = "rgb(243,243,243)",
                        paper_bgcolor = "rgb(243,243,243)",
                        xaxis = dict(gridcolor = 'rgb(255, 255, 255)',
                                     title = "metric",
                                     zerolinewidth=1,
                                     ticklen=5,gridwidth=2),
                        yaxis = dict(gridcolor = 'rgb(255, 255, 255)',
                                     zerolinewidth=1,ticklen=5,gridwidth=2),
                        margin = dict(l = 250),
                        height = 780
                       )
                  )


trace1  = output_tracer("Accuracy_score","#6699FF")
trace2  = output_tracer('Recall_score',"red")
trace3  = output_tracer('Precision',"#33CC99")
trace4  = output_tracer('f1_score',"lightgrey")
trace5  = output_tracer('Kappa_metric',"#FFCC99")

data = [trace1,trace2,trace3,trace4,trace5]
fig = go.Figure(data=data,layout=layout)
py.iplot(fig)

|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||